package com.xiaotu.spider.project.pageprocessor.list;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang.StringUtils;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import com.google.gson.Gson;
import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.FileUtil;
import com.xiaotu.common.util.KeywordEncode;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.RegexUtils;
import com.xiaotu.common.util.SepratorUtil;
import com.xiaotu.spider.AUrlPageProcessor;

import net.sf.json.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

/**
 * 豆瓣爬取类--TV 影评和评论
 */
@Component("DouBanListPageProcessor")
@Scope("prototype")
public class DouBanListPageProcessor extends AUrlPageProcessor
{
	
    private String fileName; //读取tvName文件路径
    private String fileReviews; //影评存放路径
    private String fileSubject; //电影详情存放路径
    private String movieUrl; //不匹配电影URL
    
    @Override
    protected String[] getStartRequestUrls()
    {
        List<JSONObject> targetList = readFileColumn(fileName); //读取excel电影名
        List<String> linkList = new ArrayList<String>();
        for (JSONObject target : targetList)
        {
            String tvName = target.getString("tvName");
            String startDate = target.getString("startDate");
            linkList.add("https://www.douban.com/search?cat=1002&q=" + KeywordEncode.encode(tvName) + "&startDate=" +
                    startDate);
        }
        String[] urlArr = new String[linkList.size()];
        linkList.toArray(urlArr);
        return urlArr;

    }

    @Override
    public void initRunParams(Map<String, Object> map)
    {
        if (map.get("fileName") == null)
            throw new SpiderException(
                    SpiderExceptionCode.PageProcessorParameterErrorException);
        fileName = map.get("fileName") + StringUtils.EMPTY;

        if (map.get("fileReviews") == null)
            throw new SpiderException(
                    SpiderExceptionCode.PageProcessorParameterErrorException);
        fileReviews = map.get("fileReviews") + StringUtils.EMPTY;

        if (map.get("fileSubject") == null)
            throw new SpiderException(
                    SpiderExceptionCode.PageProcessorParameterErrorException);
        fileSubject = map.get("fileSubject") + StringUtils.EMPTY;
        
        if (map.get("movieUrl") == null)
            throw new SpiderException(
                    SpiderExceptionCode.PageProcessorParameterErrorException);
        movieUrl = map.get("movieUrl") + StringUtils.EMPTY;
    }

    @Override
    public String getPageDataType(Page page)
    {
        String url = page.getUrl().toString();
        if (url.contains(
                "https://www.douban.com/search?cat=1002&q="))
            return Constants.PageDataType.SEARCH_LIST;
        else if (url.contains("https://www.douban.com/link2/"))
            return Constants.PageDataType.TV_PRO_COMMENT;
        else if (url.contains("/review"))
            return Constants.PageDataType.TV_COMMENT;
        return Constants.PageDataType.TV_PRO_COMMENT;
    }

    @Override
    public void process(Page page)
    {
        String url = page.getUrl().toString();
        Map<String, Object> tvMap = page.getRequest().getExtras();
        if (url.length() > 0 && url.contains("&q="))
        {
            String tvName = url.substring(url.indexOf("&q=") + 3, url.indexOf("&startDate"));
            String startDate = url.substring(url.indexOf("Date=") + 5, url.length());
            tvMap.put("tvName", KeywordEncode.decode(tvName));
            tvMap.put("startDate", startDate);
        }
        switch (this.getPageDataType(page))
        {
            case Constants.PageDataType.SEARCH_LIST:
                this.searchListProcess(page, tvMap);
                break;
            case Constants.PageDataType.TV_PRO_COMMENT:
                this.proCommentProcess(page, tvMap);
                break;
            case Constants.PageDataType.TV_COMMENT:
                this.commentProcess(page, tvMap);
                break;
            default:
                break;
        }
    }

    private void searchListProcess(Page page, Map<String, Object> tvMap)
    {
    	boolean logType =true;
    	String status ="1";
        //获取搜索也页面列表
        final List<String> searchListHTML = page.getHtml()
                .xpath("//div[@class='result']/div[@class='content']").all();
        for (String htmlStr : searchListHTML)
        {
            Html html = new Html(htmlStr);
            //获取影片
            String movieType = html.xpath("//div[@class='title']/h3/span/text()").toString();
            //获取影片名称
            String movieName = html.xpath("//div[@class='title']/h3/a/text()").toString().trim();
            //获取影片年代
            String movieAges = html.xpath("//div[@class='rating-info']/span[@class='subject-cast']/text()").toString();
            if (movieAges != null && movieAges != "")
            {
            	String movieAge = movieAges.substring(movieAges.length() - 4, movieAges.length());
                tvMap.put("movieAge", movieAge);
            }
            String tvMapName = (String) tvMap.get("tvName");
            tvMap.put("movieName", movieName);
            String links = html.xpath("//div[@class='title']/h3/a/@href").get();
            if (movieType.equals("[电影]") && movieName.equals(tvMapName)){
            	status ="0";
                tvMap.put("linkUrl", links);
                //存放影片地址
                PageUtils.addTargetRequest(page, links, page.getRequest().getExtras());
            }
        }
	        if(logType && status=="1"){
	        	FileUtil.writeFile(movieUrl, page.getUrl().toString()
	    				+ SepratorUtil.LINE_SEPRATOR);
	    		logType =false;
	    	}
    }


    private void proCommentProcess(Page page, Map<String, Object> tvMap){
    	StringBuffer buffer = new StringBuffer();
    		/*String movieTypeStatus = page.getHtml().xpath("//div[@id='info']").get();
    		if(!movieTypeStatus.contains("编剧") || !movieTypeStatus.contains("主演") || !movieTypeStatus.contains("又名")
    				|| !movieTypeStatus.contains("导演") || !movieTypeStatus.contains("类型") || !movieTypeStatus.contains("制片国家")
    				|| !movieTypeStatus.contains("语言") || !movieTypeStatus.contains("上映日期") || !movieTypeStatus.contains("片长")){
    			return;
    		}*/
    		//电影名称
    		String moviename = (String) tvMap.get("movieName");
    		buffer.append(moviename+SepratorUtil.SEP_PIPE);
    		//电影年份
    		String movieAge =(String)tvMap.get("movieAge");
    		buffer.append(movieAge+SepratorUtil.SEP_PIPE);
    		//Excel年份
    		String tvMapDates = (String) tvMap.get("startDate");
    		if (tvMapDates != null && tvMapDates != ""){
    			String tvMapDate = tvMapDates.substring(0, 4);
    			buffer.append(tvMapDate+SepratorUtil.SEP_PIPE);
    		}
    		
    		final List<String> moviceListHTML = page.getHtml().xpath("//div[@id='info']/span").all();
    		for (String htmlStr : moviceListHTML){
    			Html html = new Html(htmlStr);
    			String moviceContent = html.xpath("//span[@class='pl']/text()").toString();
    			if (moviceContent == null)
    			{
    				continue;
    			}
    			switch (moviceContent)
    			{
    			case "导演":
    				getMovieContent(html, buffer);
    				break;
    			case "编剧":
    				getMovieContent(html, buffer);
    				break;
    			case "主演":
    				getMovieContent(html, buffer);
    				break;
    			case "类型:":
    				List<String> movieTypes = page.getHtml().xpath("//div[@id='info']/span[@property='v:genre']/text" +
    						"()").all();
    				for (String movieType : movieTypes)
    				{
    					buffer.append(SepratorUtil.SEP_SLASH_EN + movieType);
    				}
    				buffer.substring(0, buffer.length() - 1).trim();
    				buffer.append(SepratorUtil.SEP_PIPE);
    				break;
    			case "语言:":
    				String languages = page.getHtml().xpath("//div[@id='info']").get();
    				List<String> languageRex = getSrings(languages, Pattern.quote("<span class=\"pl\">语言:</span>") +
    						"(.*?)" + Pattern.quote("<span class=\"pl\">上映日期:</span> "));
    				for (String languageStr : languageRex)
    				{
    					if (languageStr.length() >= 4)
    					{
    						String language = languageStr.substring(0, languageStr.length() - 8);
    						buffer.append(language.trim());
    					}
    				}
    				buffer.append(SepratorUtil.SEP_PIPE);
    				break;
    			case "制片国家/地区:":
    				String countrys = page.getHtml().xpath("//div[@id='info']").get();
    				List<String> countryRex = getSrings(countrys, Pattern.quote("<span class=\"pl\">制片国家/地区:</span>")
    						+ "(.*?)" + Pattern.quote("<span class=\"pl\">语言:</span>"));
    				for (String countryStr : countryRex)
    				{
    					if (countryStr.length() >= 4)
    					{
    						String country = countryStr.substring(0, countryStr.length() - 8);
    						buffer.append(country.trim());
    					}
    				}
    				buffer.append(SepratorUtil.SEP_PIPE);
    				break;
    			case "上映日期:":
    				List<String> movielens = page.getHtml().xpath
    				("//div[@id='info']/span[@property='v:initialReleaseDate']/text()").all();
    				for (String movielen : movielens)
    				{
    					buffer.append(SepratorUtil.SEP_SLASH_EN + movielen);
    				}
    				buffer.substring(0, buffer.length() - 1).trim();
    				buffer.append(SepratorUtil.SEP_PIPE);
    				break;
    			case "片长:":
    				String release = page.getHtml().xpath("//div[@id='info']/span[@property='v:runtime']/text()")
    				.toString();
    				buffer.append(SepratorUtil.SEP_SLASH_EN + release);
    				buffer.substring(0, buffer.length() - 1).trim();
    				buffer.append(SepratorUtil.SEP_PIPE);
    				break;
    			case "又名:":
    				String aliasNames = page.getHtml().xpath("//div[@id='info']").get();
    				List<String> aliasNameRex = getSrings(aliasNames, Pattern.quote("<span class=\"pl\">又名:</span>")
    						+ "(.*?)" + Pattern.quote("<span class=\"pl\">IMDb链接:</span>"));
    				for (String aliasNameStr : aliasNameRex)
    				{
    					if (aliasNameStr.length() >= 4)
    					{
    						String aliasName = aliasNameStr.substring(0, aliasNameStr.length() - 8);
    						buffer.append(aliasName.trim());
    					}
    				}
    				buffer.append(SepratorUtil.SEP_PIPE);
    				break;
    				
    			default:
    				break;
    			}
    			
    		}
    		
    		//豆瓣评分
    		String doubanscore = page.getHtml().xpath("//div[@class='rating_self clearfix']/strong[@class='ll " +
    				"rating_num']/text()").toString();
    		if(StringUtils.isEmpty(doubanscore)){
    			doubanscore =SepratorUtil.SEP_SPACE_EN;
    		}
    		buffer.append(doubanscore+SepratorUtil.SEP_PIPE);
    		//参加的评论人数
    		String ratingPeople =page.getHtml().xpath("//div[@class='rating_sum']/a/span/text()").get();
    		if(StringUtils.isEmpty(ratingPeople)){
    			ratingPeople =SepratorUtil.SEP_SPACE_EN;
    		}
    		buffer.append(ratingPeople+SepratorUtil.SEP_PIPE);
    		//短评数量
    		String commentNums = page.getHtml().xpath("//div[@id='comments-section']/div/h2/span[@class='pl']/a/text()")
    				.toString();
    		if(StringUtils.isEmpty(commentNums)){
    			commentNums =SepratorUtil.SEP_SPACE_EN;
    		}else{
    			commentNums = commentNums.substring(commentNums.indexOf("全部") + 2, commentNums.indexOf("条")).trim();
    		}
    			buffer.append(commentNums+SepratorUtil.SEP_PIPE);
    		
    		//相关问题数量
    		String relatedNums = page.getHtml().xpath("//div[@id='askmatrix']/div[@class='mod-hd']/h2/span/a/text()")
    				.toString();
    		if(StringUtils.isEmpty(relatedNums)){
    			relatedNums =SepratorUtil.SEP_SPACE_EN;
    		}else{
    			relatedNums = relatedNums.substring(relatedNums.indexOf("全部") + 2, relatedNums.indexOf("个")).trim();
    		}
    			buffer.append(relatedNums+SepratorUtil.SEP_PIPE);
    		
    		//影评数量
    		String criticsNums = page.getHtml().xpath("//header/h2/span[@class='pl']/a/text()").toString();
			if(StringUtils.isEmpty(criticsNums)){
				criticsNums =SepratorUtil.SEP_SPACE_EN;
			}else{
				criticsNums = criticsNums.substring(criticsNums.indexOf("全部") + 2, criticsNums.indexOf("条")).trim();
			}
    			buffer.append(criticsNums+SepratorUtil.SEP_PIPE);
    		
    		List<String> Stars = page.getHtml().xpath("//div[@class='ratings-on-weight']/").all();
    		if(Stars.size()>0){
    			for (String stringStr : Stars)
    			{
    				Html htmlStr = new Html(stringStr);
    				String starsType = htmlStr.xpath("//div[@class='item']/span[@class='starstop']/text()").toString().trim();
    				String movieStars = htmlStr.xpath("//div[@class='item']/span[@class='rating_per']/text()")
    						.toString();
    				if(StringUtils.isEmpty(movieStars)){
    					movieStars =SepratorUtil.SEP_SPACE_EN;
    				}
    				
    				switch (starsType)
    				{
    				case "5星":
    					buffer.append(movieStars+SepratorUtil.SEP_PIPE);
    					break;
    				case "4星":
    					buffer.append(movieStars+SepratorUtil.SEP_PIPE);
    					break;
    				case "3星":
    					buffer.append(movieStars+SepratorUtil.SEP_PIPE);
    					break;
    				case "2星":
    					buffer.append(movieStars+SepratorUtil.SEP_PIPE);
    					break;
    				case "1星":
    					buffer.append(movieStars+SepratorUtil.SEP_PIPE);
    					break;
    					
    				default:
    					break;
    				}
    			}
    		}else{
    			buffer.append(SepratorUtil.SEP_SPACE_EN+SepratorUtil.SEP_PIPE+SepratorUtil.SEP_SPACE_EN+SepratorUtil.SEP_PIPE+
    					SepratorUtil.SEP_SPACE_EN+SepratorUtil.SEP_PIPE+SepratorUtil.SEP_SPACE_EN+SepratorUtil.SEP_PIPE+SepratorUtil.SEP_SPACE_EN+SepratorUtil.SEP_PIPE);
    		}
    		
    		//影评地址
    		String movieurl =(String) tvMap.get("linkUrl");
    		buffer.append(movieurl);
    		
    		FileUtil.writeFile(fileSubject, buffer.toString()
    				+ SepratorUtil.LINE_SEPRATOR);
    		String comUrls = page.getHtml().xpath("//section[@class='reviews mod " +
    				"movie-content']/header/a/@data-verify-url").get();
    		if (comUrls.length() > 0)
    		{
    			String comUrl = comUrls.substring(comUrls.indexOf("redir=") + 6, comUrls.indexOf("new_review")) + "reviews";
    			PageUtils.addTargetRequest(page, comUrl, page.getRequest().getExtras());
    		}
}

    private void commentProcess(Page page, Map<String, Object> tvMap)
    {
        StringBuffer buffer = new StringBuffer();
        // 获取专业影评
        List<Selectable> proCommList = page.getHtml()
                .xpath("//div[@class='review-list']/div").nodes();

        if (null == proCommList || proCommList.size() < 1)
            throw new SpiderException(
                    SpiderExceptionCode.DataInvalideException);

        for (Selectable procmt : proCommList)
        {

            //电影名称
            String movieName = (String) tvMap.get("tvName");
            buffer.append(movieName);
            //电影年份
    		String movieAge =(String)tvMap.get("movieAge");
    		buffer.append(SepratorUtil.SEP_PIPE+movieAge);
            //影评发布者
            String movieReviewers = procmt.xpath("//div[@class='header-more']/a/span/text()").toString();
            if(StringUtils.isEmpty(movieReviewers)){
            	movieReviewers =SepratorUtil.SEP_SPACE_EN;
            }
            if(movieReviewers.contains("|")){
            	movieReviewers =movieReviewers.replace("|", "");
            }
            buffer.append(SepratorUtil.SEP_PIPE + movieReviewers);

            //影评发布时间
            String movieReleaseTime = procmt.xpath("//div[@class='header-more']/span[@class='main-meta']/text()")
                    .toString();
            if(StringUtils.isEmpty(movieReleaseTime)){
            	movieReleaseTime =SepratorUtil.SEP_SPACE_EN;
            }
            buffer.append(SepratorUtil.SEP_PIPE + movieReleaseTime);

            //影评标题
            String movieTitle = procmt.xpath("//h3[@class='title']/a/text()").toString();
            if(StringUtils.isEmpty(movieTitle)){
            	movieTitle =SepratorUtil.SEP_SPACE_EN;
            }
            buffer.append(SepratorUtil.SEP_PIPE + movieTitle);

            //影评星级
            String movieStar = procmt.xpath("//div[@class='header-more']/span[@property='v:rating']/@class")
                    .toString();
            if (StringUtils.isEmpty(movieStar) || StringUtils.isEmpty(movieStar.trim())){
            	movieStar =SepratorUtil.SEP_SPACE_EN;
            	buffer.append(SepratorUtil.SEP_PIPE+movieStar);
            }else{
            	movieStar = movieStar.substring(movieStar.indexOf("allstar") + 7, movieStar.indexOf("0 " +
            			"main-title-rating"));
            	buffer.append(SepratorUtil.SEP_PIPE + movieStar);
            }

            //影评内容
            String comment = procmt
                    .xpath("//div[@class='short-content']/text()").toString();
            if (StringUtils.isEmpty(comment) || StringUtils.isEmpty(comment.trim())){
            	comment ="没关系,可以显示全文";
            }
            	buffer.append(SepratorUtil.SEP_PIPE + comment);
            
            //回复评论人数
            String movieUseNum = procmt.xpath("//div[@class='main-bd']/div/div[@class='short-content']/a/text()")
                    .toString();
            if (StringUtils.isEmpty(movieUseNum) || StringUtils.isEmpty(movieUseNum.trim())){
            	movieUseNum =SepratorUtil.SEP_SPACE_EN;
            }
            buffer.append(SepratorUtil.SEP_PIPE + movieUseNum);

            //有用 没有人数
            String up = null, down = null;
            String upDown = procmt.xpath("//span[@class='left']/text()")
                    .toString();
            if (StringUtils.isNotEmpty(upDown))
            {
                up = RegexUtils.regexSearch("([0-9]+)有用", upDown);
                up = StringUtils.isEmpty(up) ? up
                        : RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER, up);
                if (StringUtils.isEmpty(up)){
                	up =SepratorUtil.SEP_SPACE_EN;
                }
                buffer.append(SepratorUtil.SEP_PIPE + up);

                down = RegexUtils.regexSearch("([0-9]+)没用", upDown);
                down = StringUtils.isEmpty(down) ? down
                        : RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER,
                        down);
                if (StringUtils.isEmpty(down)){
                	down =SepratorUtil.SEP_SPACE_EN;
                }
                buffer.append(SepratorUtil.SEP_PIPE + down);
            }

            FileUtil.writeFile(fileReviews, buffer.toString() + SepratorUtil.LINE_SEPRATOR);
            buffer.setLength(0);
        }

        int pageMax = 200;
        String current = page.getHtml()
                .xpath("//div[@class='paginator']/span[@class='next']/a/@href").get();
        if (current != null)
        {
            String pageSize = current.substring(current.indexOf("start=") + 6, current.length());
            if (StringUtils.isNotEmpty(pageSize))
            {
                pageSize = RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER, pageSize);
                if (StringUtils.isNotEmpty(pageSize))
                {
                    if (pageMax >= Integer.parseInt(pageSize))
                        PageUtils.addTargetRequest(page,
                                current,
                                page.getRequest().getExtras());
                }
            }
        }
    }

    @Override
    protected String getSpiderType()
    {
        return "DouBanList";
    }


    public static String getMovieContent(Html html, StringBuffer buffer){
        List<String> directorList = html.xpath("//span[@class='attrs']/a/text()").all();
        for (String directorString : directorList){
            buffer.append(directorString + SepratorUtil.SEP_SLASH_EN);
        }
        buffer.append(SepratorUtil.SEP_PIPE);
        String buffers = buffer.substring(0, buffer.length() - 1).trim();
        return buffers.toString();
    }

    public static String getMovieType(Html html, StringBuffer buffer)
    {
        List<String> directorList = html.xpath("//span[@property='v:genre']/text()").all();
        for (String directorString : directorList)
        {
            buffer.append(directorString + SepratorUtil.SEP_SLASH_EN);
        }
        buffer.append(SepratorUtil.SEP_PIPE);
        String buffers = buffer.substring(0, buffer.length() - 1).trim();
        return buffers.toString();
    }

    /**
     * 获取多个目标字符串
     */
    public static List<String> getSrings(String input, String regex)
    {
        List<String> result = new LinkedList<String>();
        Pattern pat = Pattern.compile(regex, Pattern.DOTALL);
        Matcher mat = pat.matcher(input);
        while (mat.find())
        {
            result.add(mat.group(1));
        }
        return result;
    }

    /**
     * 读取文件指定列
     *
     * @param fileName 文件名称
     * @return 行列表
     */
    public static List<JSONObject> readFileColumn(String fileName)
    {
        Map<String, Object> map = new HashMap<String, Object>();
        Workbook wb = null;
        String json;
        List<JSONObject> list = new ArrayList<JSONObject>();
        try
        {
            InputStream inputStream = new FileInputStream(fileName);
            if (fileName.endsWith("xls"))
            {
                wb = new HSSFWorkbook(inputStream); // 解析xls格式
            } else if (fileName.endsWith("xlsx"))
            {
                wb = new XSSFWorkbook(inputStream); // 解析xlsx格式
            }
            Sheet sheet = wb.getSheetAt(0); // 第一个工作表

            int firstRowNum = 1;
            int lastRowNum = sheet.getLastRowNum();

            Row row = null;
            Cell cell_a1 = null;
            Cell cell_a5 = null;
            for (int i = firstRowNum; i <= lastRowNum; i++)
            {
                row = sheet.getRow(i);          //取得第i行
                cell_a1 = row.getCell(1);      //取得i行的第一列
                cell_a5 = row.getCell(5);
                String cellValue1 = cell_a1.getStringCellValue().trim();
                String cellValue5 = cell_a5.getStringCellValue().trim();
                map.put("tvName", cellValue1);
                map.put("startDate", cellValue5);
                json = new Gson().toJson(map);
                JSONObject jsStr = JSONObject.fromObject(json);
                list.add(jsStr);
            }

        }
        catch (FileNotFoundException e)
        {
            e.printStackTrace();
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }
        return list;
    }

}
